{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fd94b946",
   "metadata": {},
   "source": [
    "### 本周内容\n",
    "· 2.1 [Request-HTML]\n",
    "\n",
    "· 2.1.1 Request-Html体验\n",
    "\n",
    "· 2.1.2 体验项目二：获取页面文本\n",
    "\n",
    "· 2.1.3 体验项目三：获取页面图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ba384573",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not find a version that satisfies the requirement request-html (from versions: none)\n",
      "ERROR: No matching distribution found for request-html\n"
     ]
    }
   ],
   "source": [
    "!pip install request-html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c6c7be74",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Usage:   \n",
      "  pip install [options] <requirement specifier> [package-index-options] ...\n",
      "  pip install [options] -r <requirements file> [package-index-options] ...\n",
      "  pip install [options] [-e] <vcs project url> ...\n",
      "  pip install [options] [-e] <local project path> ...\n",
      "  pip install [options] <archive url/path> ...\n",
      "\n",
      "no such option: -u\n"
     ]
    }
   ],
   "source": [
    "!pip install -upgrade pip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "28956504",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in d:\\program files (x86)\\anaconda3\\lib\\site-packages (23.0.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade pip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ee623966",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests-html in d:\\program files (x86)\\anaconda3\\lib\\site-packages (0.10.0)\n",
      "Requirement already satisfied: requests in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (2.25.1)\n",
      "Requirement already satisfied: fake-useragent in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (1.1.1)\n",
      "Requirement already satisfied: pyquery in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (2.0.0)\n",
      "Requirement already satisfied: w3lib in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (2.1.1)\n",
      "Requirement already satisfied: bs4 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (0.0.1)\n",
      "Requirement already satisfied: pyppeteer>=0.0.14 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (1.0.2)\n",
      "Requirement already satisfied: parse in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests-html) (1.19.0)\n",
      "Requirement already satisfied: pyee<9.0.0,>=8.1.0 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (8.2.2)\n",
      "Requirement already satisfied: websockets<11.0,>=10.0 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (10.4)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.42.1 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (4.59.0)\n",
      "Requirement already satisfied: certifi>=2021 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (2022.12.7)\n",
      "Requirement already satisfied: appdirs<2.0.0,>=1.4.3 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (1.4.4)\n",
      "Requirement already satisfied: urllib3<2.0.0,>=1.25.8 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (1.26.4)\n",
      "Requirement already satisfied: importlib-metadata>=1.4 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyppeteer>=0.0.14->requests-html) (3.10.0)\n",
      "Requirement already satisfied: beautifulsoup4 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from bs4->requests-html) (4.9.3)\n",
      "Requirement already satisfied: importlib-resources>=5.0 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from fake-useragent->requests-html) (5.12.0)\n",
      "Requirement already satisfied: cssselect>=1.2.0 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyquery->requests-html) (1.2.0)\n",
      "Requirement already satisfied: lxml>=2.1 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from pyquery->requests-html) (4.6.3)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests->requests-html) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from requests->requests-html) (2.10)\n",
      "Requirement already satisfied: zipp>=0.5 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests-html) (3.4.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in d:\\program files (x86)\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4->requests-html) (2.2.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install requests-html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "86cc73eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession\n",
    "\n",
    "session =HTMLSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "831a90ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = session.get('https://python.org/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1baa2610",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a263e786",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Connection': 'keep-alive', 'Content-Length': '50059', 'Server': 'nginx', 'Content-Type': 'text/html; charset=utf-8', 'X-Frame-Options': 'DENY', 'Via': '1.1 vegur, 1.1 varnish, 1.1 varnish', 'Accept-Ranges': 'bytes', 'Date': 'Fri, 10 Mar 2023 07:55:11 GMT', 'Age': '2649', 'X-Served-By': 'cache-iad-kiad7000025-IAD, cache-nrt-rjtf7700078-NRT', 'X-Cache': 'HIT, HIT', 'X-Cache-Hits': '474, 1750', 'X-Timer': 'S1678434911.255891,VS0,VE0', 'Vary': 'Cookie', 'Strict-Transport-Security': 'max-age=63072000; includeSubDomains; preload'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d29c56fb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'window.dataLayer = window.dataLayer || []; function gtag(){dataLayer.push(arguments);} gtag(\\'js\\', new Date()); gtag(\\'config\\', \\'G-TF35YF9CVH\\');\\nWelcome to Python.org\\n{ \"@context\": \"https://schema.org\", \"@type\": \"WebSite\", \"url\": \"https://www.python.org/\", \"potentialAction\": { \"@type\": \"SearchAction\", \"target\": \"https://www.python.org/search/?q={search_term_string}\", \"query-input\": \"required name=search_term_string\" } } var _gaq = _gaq || []; _gaq.push([\\'_setAccount\\', \\'UA-39055973-1\\']); _gaq.push([\\'_trackPageview\\']); (function() { var ga = document.createElement(\\'script\\'); ga.type = \\'text/javascript\\'; ga.async = true; ga.src = (\\'https:\\' == document.location.protocol ? \\'https://ssl\\' : \\'http://www\\') + \\'.google-analytics.com/ga.js\\'; var s = document.getElementsByTagName(\\'script\\')[0]; s.parentNode.insertBefore(ga, s); })();\\nNotice: While JavaScript is not essential for this website, your interaction with the content will be limited. Please turn JavaScript on for the full experience.\\nSkip to content\\n▼ Close\\nPython\\nPSF\\nDocs\\nPyPI\\nJobs\\nCommunity\\n▲ The Python Network\\nDonate\\n≡ Menu\\nSearch This Site GO\\nA A\\nSmaller\\nLarger\\nReset\\nSocialize\\nFacebook\\nTwitter\\nChat on IRC\\nAbout\\nApplications\\nQuotes\\nGetting Started\\nHelp\\nPython Brochure\\nDownloads\\nAll releases\\nSource code\\nWindows\\nmacOS\\nOther Platforms\\nLicense\\nAlternative Implementations\\nDocumentation\\nDocs\\nAudio/Visual Talks\\nBeginner\\'s Guide\\nDeveloper\\'s Guide\\nFAQ\\nNon-English Docs\\nPEP Index\\nPython Books\\nPython Essays\\nCommunity\\nDiversity\\nMailing Lists\\nIRC\\nForums\\nPSF Annual Impact Report\\nPython Conferences\\nSpecial Interest Groups\\nPython Logo\\nPython Wiki\\nCode of Conduct\\nCommunity Awards\\nGet Involved\\nShared Stories\\nSuccess Stories\\nArts\\nBusiness\\nEducation\\nEngineering\\nGovernment\\nScientific\\nSoftware Development\\nNews\\nPython News\\nPSF Newsletter\\nPSF News\\nPyCon US News\\nNews from the Community\\nEvents\\nPython Events\\nUser Group Events\\nPython Events Archive\\nUser Group Events Archive\\nSubmit an Event\\n>_ Launch Interactive Shell\\n# Python 3: Fibonacci series up to n >>> def fib(n): >>> a, b = 0, 1 >>> while a < n: >>> print(a, end=\\' \\') >>> a, b = b, a+b >>> print() >>> fib(1000) 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987\\nFunctions Defined\\nThe core of extensible programming is defining functions. Python allows mandatory and optional arguments, keyword arguments, and even arbitrary argument lists. More about defining functions in Python\\xa03\\n# Python 3: List comprehensions >>> fruits = [\\'Banana\\', \\'Apple\\', \\'Lime\\'] >>> loud_fruits = [fruit.upper() for fruit in fruits] >>> print(loud_fruits) [\\'BANANA\\', \\'APPLE\\', \\'LIME\\'] # List and the enumerate function >>> list(enumerate(fruits)) [(0, \\'Banana\\'), (1, \\'Apple\\'), (2, \\'Lime\\')]\\nCompound Data Types\\nLists (known as arrays in other languages) are one of the compound data types that Python understands. Lists can be indexed, sliced and manipulated with other built-in functions. More about lists in Python\\xa03\\n# Python 3: Simple arithmetic >>> 1 / 2 0.5 >>> 2 ** 3 8 >>> 17 / 3 # classic division returns a float 5.666666666666667 >>> 17 // 3 # floor division 5\\nIntuitive Interpretation\\nCalculations are simple with Python, and expression syntax is straightforward: the operators +, -, * and / work as expected; parentheses () can be used for grouping. More about simple math functions in Python\\xa03.\\n# For loop on a list >>> numbers = [2, 4, 6, 8] >>> product = 1 >>> for number in numbers: ... product = product * number ... >>> print(\\'The product is:\\', product) The product is: 384\\nAll the Flow You’d Expect\\nPython knows the usual control flow statements that other languages speak — if, for, while and range — with some of its own twists, of course. More control flow tools in Python\\xa03\\n# Simple output (with Unicode) >>> print(\"Hello, I\\'m Python!\") Hello, I\\'m Python! # Input, assignment >>> name = input(\\'What is your name?\\\\n\\') What is your name? Python >>> print(f\\'Hi, {name}.\\') Hi, Python.\\nQuick & Easy to Learn\\nExperienced programmers in any other language can pick up Python very quickly, and beginners find the clean syntax and indentation structure easy to learn. Whet your appetite with our Python\\xa03 overview.\\nPython is a programming language that lets you work quickly and integrate systems more effectively. Learn More\\nGet Started\\nWhether you\\'re new to programming or an experienced developer, it\\'s easy to learn and use Python.\\nStart with our Beginner’s Guide\\nDownload\\nPython source code and installers are available for download for all versions!\\nLatest: Python 3.11.2\\nDocs\\nDocumentation for Python\\'s standard library, along with tutorials and guides, are available online.\\ndocs.python.org\\nJobs\\nLooking for work or have a Python related position that you\\'re trying to hire for? Our relaunched community-run job board is the place to go.\\njobs.python.org\\nLatest News\\nMore\\n2023-03-08 Python 3.12.0 alpha 6 released\\n2023-02-15 The Case for a Second Developer-in-Residence for Python\\n2023-02-10 Python Packaging Strategy Discussion Summary - Part 1\\n2023-02-08 Python 3.11.2, Python 3.10.10 and 3.12.0 alpha 5 are available\\n2023-02-08 Announcing Python Software Foundation Fellow Members for Q4 2022! 🎉\\nUpcoming Events\\nMore\\n2023-03-13 Python Web Conf 2023\\n2023-03-15 The long white computing cloud - NZPUG-Auckland\\n2023-03-22 Python Meeting Düsseldorf\\n2023-04-01 PyTexas 2023\\n2023-04-07 PyCamp Spain 2023\\nSuccess Stories\\nMore\\nPython powers major aspects of Abridge’s ML lifecycle, including data annotation, research and experimentation, and ML model deployment to production.\\nAbridging clinical conversations using Python by Nimshi Venkat and Sandeep Konam\\nUse Python for…\\nMore\\nWeb Development: Django, Pyramid, Bottle, Tornado, Flask, web2py\\nGUI Development: tkInter, PyGObject, PyQt, PySide, Kivy, wxPython\\nScientific and Numeric: SciPy, Pandas, IPython\\nSoftware Development: Buildbot, Trac, Roundup\\nSystem Administration: Ansible, Salt, OpenStack, xonsh\\n>>> Python Enhancement Proposals (PEPs): The future of Python is discussed here. RSS\\n>>> Python Software Foundation\\nThe mission of the Python Software Foundation is to promote, protect, and advance the Python programming language, and to support and facilitate the growth of a diverse and international community of Python programmers. Learn more\\nBecome a Member Donate to the PSF\\n▲ Back to Top\\nAbout\\nApplications\\nQuotes\\nGetting Started\\nHelp\\nPython Brochure\\nDownloads\\nAll releases\\nSource code\\nWindows\\nmacOS\\nOther Platforms\\nLicense\\nAlternative Implementations\\nDocumentation\\nDocs\\nAudio/Visual Talks\\nBeginner\\'s Guide\\nDeveloper\\'s Guide\\nFAQ\\nNon-English Docs\\nPEP Index\\nPython Books\\nPython Essays\\nCommunity\\nDiversity\\nMailing Lists\\nIRC\\nForums\\nPSF Annual Impact Report\\nPython Conferences\\nSpecial Interest Groups\\nPython Logo\\nPython Wiki\\nCode of Conduct\\nCommunity Awards\\nGet Involved\\nShared Stories\\nSuccess Stories\\nArts\\nBusiness\\nEducation\\nEngineering\\nGovernment\\nScientific\\nSoftware Development\\nNews\\nPython News\\nPSF Newsletter\\nPSF News\\nPyCon US News\\nNews from the Community\\nEvents\\nPython Events\\nUser Group Events\\nPython Events Archive\\nUser Group Events Archive\\nSubmit an Event\\nContributing\\nDeveloper\\'s Guide\\nIssue Tracker\\npython-dev list\\nCore Mentorship\\nReport a Security Issue\\n▲ Back to Top\\nHelp & General Contact\\nDiversity Initiatives\\nSubmit Website Bug\\nStatus\\nCopyright ©2001-2023. \\xa0Python Software Foundation \\xa0Legal Statements \\xa0Privacy Policy \\xa0Powered by Heroku\\nwindow.jQuery || document.write(\\'<script src=\"/static/js/libs/jquery-1.8.2.min.js\"><\\\\/script>\\') window.jQuery || document.write(\\'<script src=\"/static/js/libs/jquery-ui-1.12.1.min.js\"><\\\\/script>\\')'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e031d758",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'/',\n",
       " '//docs.python.org/3/tutorial/',\n",
       " '//docs.python.org/3/tutorial/controlflow.html',\n",
       " '//docs.python.org/3/tutorial/controlflow.html#defining-functions',\n",
       " '//docs.python.org/3/tutorial/introduction.html#lists',\n",
       " '//jobs.python.org',\n",
       " '/about/',\n",
       " '/about/apps',\n",
       " '/about/apps/',\n",
       " '/about/gettingstarted/',\n",
       " '/about/help/',\n",
       " '/about/legal/',\n",
       " '/about/quotes/',\n",
       " '/blogs/',\n",
       " '/community-landing/',\n",
       " '/community/',\n",
       " '/community/awards',\n",
       " '/community/diversity/',\n",
       " '/community/forums/',\n",
       " '/community/irc/',\n",
       " '/community/lists/',\n",
       " '/community/logos/',\n",
       " '/community/sigs/',\n",
       " '/community/workshops/',\n",
       " '/dev/',\n",
       " '/dev/core-mentorship/',\n",
       " '/dev/peps/',\n",
       " '/dev/peps/peps.rss',\n",
       " '/dev/security/',\n",
       " '/doc/',\n",
       " '/doc/av',\n",
       " '/doc/essays/',\n",
       " '/download/alternatives',\n",
       " '/download/other/',\n",
       " '/downloads/',\n",
       " '/downloads/macos/',\n",
       " '/downloads/release/python-3112/',\n",
       " '/downloads/source/',\n",
       " '/downloads/windows/',\n",
       " '/events/',\n",
       " '/events/calendars/',\n",
       " '/events/python-events/',\n",
       " '/events/python-events/1436/',\n",
       " '/events/python-events/1454/',\n",
       " '/events/python-events/past/',\n",
       " '/events/python-user-group/',\n",
       " '/events/python-user-group/1431/',\n",
       " '/events/python-user-group/1435/',\n",
       " '/events/python-user-group/1458/',\n",
       " '/events/python-user-group/past/',\n",
       " '/jobs/',\n",
       " '/privacy/',\n",
       " '/psf-landing/',\n",
       " '/psf/',\n",
       " '/psf/annual-report/2021/',\n",
       " '/psf/community-stories/',\n",
       " '/psf/conduct/',\n",
       " '/psf/donations/',\n",
       " '/psf/get-involved/',\n",
       " '/psf/newsletter/',\n",
       " '/psf/sponsorship/sponsors/#heroku',\n",
       " '/shell/',\n",
       " '/success-stories/',\n",
       " '/success-stories/abridging-clinical-conversations-using-python/',\n",
       " '/success-stories/category/arts/',\n",
       " '/success-stories/category/business/',\n",
       " '/success-stories/category/education/',\n",
       " '/success-stories/category/engineering/',\n",
       " '/success-stories/category/government/',\n",
       " '/success-stories/category/scientific/',\n",
       " '/success-stories/category/software-development/',\n",
       " '/users/membership/',\n",
       " 'http://bottlepy.org',\n",
       " 'http://brochure.getpython.info/',\n",
       " 'http://buildbot.net/',\n",
       " 'http://docs.python.org/3/tutorial/introduction.html#using-python-as-a-calculator',\n",
       " 'http://flask.pocoo.org/',\n",
       " 'http://ipython.org',\n",
       " 'http://pandas.pydata.org/',\n",
       " 'http://planetpython.org/',\n",
       " 'http://pycon.blogspot.com/',\n",
       " 'http://pyfound.blogspot.com/',\n",
       " 'http://python.org/dev/peps/',\n",
       " 'http://roundup.sourceforge.net/',\n",
       " 'http://tornadoweb.org',\n",
       " 'http://trac.edgewall.org/',\n",
       " 'http://wiki.python.org/moin/Languages',\n",
       " 'http://wiki.python.org/moin/TkInter',\n",
       " 'http://www.ansible.com',\n",
       " 'http://www.djangoproject.com/',\n",
       " 'http://www.pylonsproject.org/',\n",
       " 'http://www.riverbankcomputing.co.uk/software/pyqt/intro',\n",
       " 'http://www.scipy.org',\n",
       " 'http://www.web2py.com/',\n",
       " 'http://www.wxpython.org/',\n",
       " 'https://blog.python.org',\n",
       " 'https://bugs.python.org/',\n",
       " 'https://devguide.python.org/',\n",
       " 'https://docs.python.org',\n",
       " 'https://docs.python.org/3/license.html',\n",
       " 'https://docs.python.org/faq/',\n",
       " 'https://github.com/python/pythondotorg/issues',\n",
       " 'https://kivy.org/',\n",
       " 'https://mail.python.org/mailman/listinfo/python-dev',\n",
       " 'https://psfmember.org/civicrm/contribute/transact?reset=1&id=2',\n",
       " 'https://pyfound.blogspot.com/2023/02/announcing-python-software-foundation.html',\n",
       " 'https://pyfound.blogspot.com/2023/02/python-packaging-strategy-discussion.html',\n",
       " 'https://pyfound.blogspot.com/2023/02/the-case-for-second-developer-in.html',\n",
       " 'https://pypi.org/',\n",
       " 'https://pythoninsider.blogspot.com/2023/02/python-3112-python-31010-and-3120-alpha.html',\n",
       " 'https://pythoninsider.blogspot.com/2023/03/python-3120-alpha-6-released.html',\n",
       " 'https://saltproject.io',\n",
       " 'https://status.python.org/',\n",
       " 'https://twitter.com/ThePSF',\n",
       " 'https://wiki.gnome.org/Projects/PyGObject',\n",
       " 'https://wiki.python.org/moin/',\n",
       " 'https://wiki.python.org/moin/BeginnersGuide',\n",
       " 'https://wiki.python.org/moin/PythonBooks',\n",
       " 'https://wiki.python.org/moin/PythonEventsCalendar#Submitting_an_Event',\n",
       " 'https://wiki.qt.io/PySide',\n",
       " 'https://www.facebook.com/pythonlang?fref=ts',\n",
       " 'https://www.openstack.org',\n",
       " 'https://www.python.org/psf/',\n",
       " 'https://xon.sh'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.html.links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aa50a368",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "About\n",
      "Applications\n",
      "Quotes\n",
      "Getting Started\n",
      "Help\n",
      "Python Brochure\n"
     ]
    }
   ],
   "source": [
    "about = r.html.find('#about',first=True)\n",
    "print(about.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c808c79e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 'about', 'class': ('tier-1', 'element-1'), 'aria-haspopup': 'true'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#about的意义\n",
    "about.attrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7c774bc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<li id=\"about\" class=\"tier-1 element-1\" aria-haspopup=\"true\">\\n<a href=\"/about/\" title=\"\" class=\"\">About</a>\\n<ul class=\"subnav menu\" role=\"menu\" aria-hidden=\"true\">\\n<li class=\"tier-2 element-1\" role=\"treeitem\"><a href=\"/about/apps/\" title=\"\">Applications</a></li>\\n<li class=\"tier-2 element-2\" role=\"treeitem\"><a href=\"/about/quotes/\" title=\"\">Quotes</a></li>\\n<li class=\"tier-2 element-3\" role=\"treeitem\"><a href=\"/about/gettingstarted/\" title=\"\">Getting Started</a></li>\\n<li class=\"tier-2 element-4\" role=\"treeitem\"><a href=\"/about/help/\" title=\"\">Help</a></li>\\n<li class=\"tier-2 element-5\" role=\"treeitem\"><a href=\"http://brochure.getpython.info/\" title=\"\">Python Brochure</a></li>\\n</ul>\\n</li>'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "487cd752",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'li'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2c0651f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Element 'a' href='/about/' title='' class=()>,\n",
       " <Element 'a' href='/about/apps/' title=''>,\n",
       " <Element 'a' href='/about/quotes/' title=''>,\n",
       " <Element 'a' href='/about/gettingstarted/' title=''>,\n",
       " <Element 'a' href='/about/help/' title=''>,\n",
       " <Element 'a' href='http://brochure.getpython.info/' title=''>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#tag的意义\n",
    "about.find('a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4a0b9496",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'http://brochure.getpython.info/',\n",
       " 'https://www.python.org/about/',\n",
       " 'https://www.python.org/about/apps/',\n",
       " 'https://www.python.org/about/gettingstarted/',\n",
       " 'https://www.python.org/about/help/',\n",
       " 'https://www.python.org/about/quotes/'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "about.absolute_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dbd8a975",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'programming'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#查找某一个文本信息是否存在页面\n",
    "r.html.search('Python is a {} language')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "05072f20",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OpenAI大幅降价ChatGPT：或导致人工智能行业走向垄断\n",
      "{'https://news.cnblogs.com/n/738002/'}\n",
      "ChatGPT黑化版来了！拉踩用户智商、写暴力故事，bug只因3个字母\n",
      "{'https://news.cnblogs.com/n/737949/'}\n",
      "Nature凌晨发稿：物理层巨大突破 21℃实现室温超导\n",
      "{'https://news.cnblogs.com/n/737907/'}\n",
      "ChatGPT版机械佛Hotoke AI爆火出圈，上线5天解决13000+烦恼\n",
      "{'https://news.cnblogs.com/n/737874/'}\n",
      "一人改代码搞崩推特，马斯克气疯：全部重写！\n",
      "{'https://news.cnblogs.com/n/737816/'}\n",
      "大神李沐被曝离职！投身大模型创业，GitHub项目已开\n",
      "{'https://news.cnblogs.com/n/737809/'}\n",
      "微软杀疯了！几行代码创建私人定制ChatGPT，AI办公软件帝国来了\n",
      "{'https://news.cnblogs.com/n/737793/'}\n",
      "便宜顺畅、中文友好……自打用上Notion AI，ChatGPT都不香了！\n",
      "{'https://news.cnblogs.com/n/737780/'}\n",
      "“AI读脑术”：大脑视觉信号被Stable Diffusion复现图像\n",
      "{'https://news.cnblogs.com/n/737715/'}\n",
      "日本国土交通部在Unity商店发布“全日本城市3D模型”，全部免费用！\n",
      "{'https://news.cnblogs.com/n/737610/'}\n",
      "aardio 编程语言作者：因妻子患癌，再无精力维护项目\n",
      "{'https://news.cnblogs.com/n/737591/'}\n",
      "阿里平头哥披露RISC-V商业生态全景，已完成主流操作系统全适配\n",
      "{'https://news.cnblogs.com/n/737548/'}\n",
      "Visual Studio 2022 17.6 发布首个预览版\n",
      "{'https://news.cnblogs.com/n/737538/'}\n",
      "ChatGPT官方API发布：输出100万词只要2.7美金，成本砍掉90%\n",
      "{'https://news.cnblogs.com/n/737522/'}\n",
      "Google 面对微软的 ChatGPT 之战，可能注定失败\n",
      "{'https://news.cnblogs.com/n/737449/'}\n",
      "未来计算机在脑细胞上运行？“类器官智能”计划和路线图公布\n",
      "{'https://news.cnblogs.com/n/737432/'}\n",
      "用量子理论凭空生成能量：物理学家学会使用零点能\n",
      "{'https://news.cnblogs.com/n/737431/'}\n",
      "小扎、马斯克宣战ChatGPT！Meta和推特组建顶级AI团队，硅谷硝烟四起\n",
      "{'https://news.cnblogs.com/n/737394/'}\n"
     ]
    }
   ],
   "source": [
    "r = session.get(\"https://news.cnblogs.com/n/recommend\")\n",
    "\n",
    "# 通过css找到新闻标签\n",
    "sel ='h2.news_entry > a'\n",
    "news = r.html.find(sel)\n",
    "# print(news[1].text)\n",
    "for new in news:\n",
    "    print(new.text)\n",
    "    print(new.absolute_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "edf85952",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://news.cnblogs.com/n/recommend?page=0\n",
      "https://news.cnblogs.com/n/recommend?page=1\n",
      "https://news.cnblogs.com/n/recommend?page=2\n",
      "https://news.cnblogs.com/n/recommend?page=3\n",
      "https://news.cnblogs.com/n/recommend?page=4\n",
      "https://news.cnblogs.com/n/recommend?page=5\n",
      "https://news.cnblogs.com/n/recommend?page=6\n",
      "https://news.cnblogs.com/n/recommend?page=7\n",
      "https://news.cnblogs.com/n/recommend?page=8\n",
      "https://news.cnblogs.com/n/recommend?page=9\n",
      "https://news.cnblogs.com/n/recommend?page=10\n",
      "https://news.cnblogs.com/n/recommend?page=11\n",
      "https://news.cnblogs.com/n/recommend?page=12\n",
      "https://news.cnblogs.com/n/recommend?page=13\n",
      "https://news.cnblogs.com/n/recommend?page=14\n",
      "https://news.cnblogs.com/n/recommend?page=15\n",
      "https://news.cnblogs.com/n/recommend?page=16\n",
      "https://news.cnblogs.com/n/recommend?page=17\n",
      "https://news.cnblogs.com/n/recommend?page=18\n",
      "https://news.cnblogs.com/n/recommend?page=19\n",
      "https://news.cnblogs.com/n/recommend?page=20\n",
      "https://news.cnblogs.com/n/recommend?page=21\n",
      "https://news.cnblogs.com/n/recommend?page=22\n",
      "https://news.cnblogs.com/n/recommend?page=23\n",
      "https://news.cnblogs.com/n/recommend?page=24\n",
      "https://news.cnblogs.com/n/recommend?page=25\n",
      "https://news.cnblogs.com/n/recommend?page=26\n",
      "https://news.cnblogs.com/n/recommend?page=27\n",
      "https://news.cnblogs.com/n/recommend?page=28\n",
      "https://news.cnblogs.com/n/recommend?page=29\n",
      "https://news.cnblogs.com/n/recommend?page=30\n",
      "https://news.cnblogs.com/n/recommend?page=31\n",
      "https://news.cnblogs.com/n/recommend?page=32\n",
      "https://news.cnblogs.com/n/recommend?page=33\n",
      "https://news.cnblogs.com/n/recommend?page=34\n",
      "https://news.cnblogs.com/n/recommend?page=35\n",
      "https://news.cnblogs.com/n/recommend?page=36\n",
      "https://news.cnblogs.com/n/recommend?page=37\n",
      "https://news.cnblogs.com/n/recommend?page=38\n",
      "https://news.cnblogs.com/n/recommend?page=39\n",
      "https://news.cnblogs.com/n/recommend?page=40\n",
      "https://news.cnblogs.com/n/recommend?page=41\n",
      "https://news.cnblogs.com/n/recommend?page=42\n",
      "https://news.cnblogs.com/n/recommend?page=43\n",
      "https://news.cnblogs.com/n/recommend?page=44\n",
      "https://news.cnblogs.com/n/recommend?page=45\n",
      "https://news.cnblogs.com/n/recommend?page=46\n",
      "https://news.cnblogs.com/n/recommend?page=47\n",
      "https://news.cnblogs.com/n/recommend?page=48\n",
      "https://news.cnblogs.com/n/recommend?page=49\n",
      "https://news.cnblogs.com/n/recommend?page=50\n",
      "https://news.cnblogs.com/n/recommend?page=51\n",
      "https://news.cnblogs.com/n/recommend?page=52\n",
      "https://news.cnblogs.com/n/recommend?page=53\n",
      "https://news.cnblogs.com/n/recommend?page=54\n",
      "https://news.cnblogs.com/n/recommend?page=55\n",
      "https://news.cnblogs.com/n/recommend?page=56\n",
      "https://news.cnblogs.com/n/recommend?page=57\n",
      "https://news.cnblogs.com/n/recommend?page=58\n",
      "https://news.cnblogs.com/n/recommend?page=59\n",
      "https://news.cnblogs.com/n/recommend?page=60\n",
      "https://news.cnblogs.com/n/recommend?page=61\n",
      "https://news.cnblogs.com/n/recommend?page=62\n",
      "https://news.cnblogs.com/n/recommend?page=63\n",
      "https://news.cnblogs.com/n/recommend?page=64\n",
      "https://news.cnblogs.com/n/recommend?page=65\n",
      "https://news.cnblogs.com/n/recommend?page=66\n",
      "https://news.cnblogs.com/n/recommend?page=67\n",
      "https://news.cnblogs.com/n/recommend?page=68\n",
      "https://news.cnblogs.com/n/recommend?page=69\n",
      "https://news.cnblogs.com/n/recommend?page=70\n",
      "https://news.cnblogs.com/n/recommend?page=71\n",
      "https://news.cnblogs.com/n/recommend?page=72\n",
      "https://news.cnblogs.com/n/recommend?page=73\n",
      "https://news.cnblogs.com/n/recommend?page=74\n",
      "https://news.cnblogs.com/n/recommend?page=75\n",
      "https://news.cnblogs.com/n/recommend?page=76\n",
      "https://news.cnblogs.com/n/recommend?page=77\n",
      "https://news.cnblogs.com/n/recommend?page=78\n",
      "https://news.cnblogs.com/n/recommend?page=79\n",
      "https://news.cnblogs.com/n/recommend?page=80\n",
      "https://news.cnblogs.com/n/recommend?page=81\n",
      "https://news.cnblogs.com/n/recommend?page=82\n",
      "https://news.cnblogs.com/n/recommend?page=83\n",
      "https://news.cnblogs.com/n/recommend?page=84\n",
      "https://news.cnblogs.com/n/recommend?page=85\n",
      "https://news.cnblogs.com/n/recommend?page=86\n",
      "https://news.cnblogs.com/n/recommend?page=87\n",
      "https://news.cnblogs.com/n/recommend?page=88\n",
      "https://news.cnblogs.com/n/recommend?page=89\n",
      "https://news.cnblogs.com/n/recommend?page=90\n",
      "https://news.cnblogs.com/n/recommend?page=91\n",
      "https://news.cnblogs.com/n/recommend?page=92\n",
      "https://news.cnblogs.com/n/recommend?page=93\n",
      "https://news.cnblogs.com/n/recommend?page=94\n",
      "https://news.cnblogs.com/n/recommend?page=95\n",
      "https://news.cnblogs.com/n/recommend?page=96\n",
      "https://news.cnblogs.com/n/recommend?page=97\n",
      "https://news.cnblogs.com/n/recommend?page=98\n",
      "https://news.cnblogs.com/n/recommend?page=99\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#练习：翻页\n",
    "url_队列 = []\n",
    "for i in range(100):\n",
    "    print('https://news.cnblogs.com/n/recommend?page='+str(i))\n",
    "url_队列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b9601fcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in url_队列:\n",
    "    r = session.get(i)\n",
    "# 通过css找到新闻标签\n",
    "    sel ='div.content'\n",
    "    news = r.html.find(sel)\n",
    "# print(news[1].text)\n",
    "    print(\"------------博客园新闻_第{page}页-----------\".format(page = i.split('=')[1]))\n",
    "    for new in news:\n",
    "        print()\n",
    "        print(new.text)\n",
    "        print(new.absolute_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cebd15e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "994aa730",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cf07c20",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54541d68",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e595cdc3",
   "metadata": {},
   "source": [
    "### 4.体验项目二：获取页面文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cb3b2fa",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
